

<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<head>
  <meta charset="utf-8" />
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
  
  <title>mindspore.nn.wrap.cell_wrapper &mdash; MindSpore master documentation</title>
  

  
  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />

  
  

  
  

  

  
  <!--[if lt IE 9]>
    <script src="../../../../_static/js/html5shiv.min.js"></script>
  <![endif]-->
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
        <script src="../../../../_static/jquery.js"></script>
        <script src="../../../../_static/underscore.js"></script>
        <script src="../../../../_static/doctools.js"></script>
        <script src="../../../../_static/language_data.js"></script>
        <script async="async" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS-MML_HTMLorMML"></script>
    
    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>

    
    <link rel="index" title="Index" href="../../../../genindex.html" />
    <link rel="search" title="Search" href="../../../../search.html" /> 
</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
          

          
            <a href="../../../../index.html" class="icon icon-home"> MindSpore
          

          
          </a>

          
            
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        
        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption"><span class="caption-text">MindSpore Python API</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.html">mindspore</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.common.initializer.html">mindspore.common.initializer</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.communication.html">mindspore.communication</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.compression.html">mindspore.compression</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.context.html">mindspore.context</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.dataset.html">mindspore.dataset</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.dataset.audio.html">mindspore.dataset.audio</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.dataset.config.html">mindspore.dataset.config</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.dataset.text.html">mindspore.dataset.text</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.dataset.transforms.html">mindspore.dataset.transforms</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.dataset.vision.html">mindspore.dataset.vision</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.mindrecord.html">mindspore.mindrecord</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.nn.html">mindspore.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.nn.probability.html">mindspore.nn.probability</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.nn.transformer.html">mindspore.nn.transformer</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.numpy.html">mindspore.numpy</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.ops.html">mindspore.ops</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.parallel.html">mindspore.parallel</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.parallel.nn.html">mindspore.parallel.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.profiler.html">mindspore.profiler</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.scipy.html">mindspore.scipy</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.train.html">mindspore.train</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.boost.html">mindspore.boost</a></li>
</ul>
<p class="caption"><span class="caption-text">MindSpore C++ API</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://www.mindspore.cn/lite/api/zh-CN/master/api_cpp/mindspore.html">MindSpore Lite↗</a></li>
</ul>

            
          
        </div>
        
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../../../../index.html">MindSpore</a>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        
          

















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../../../../index.html" class="icon icon-home"></a> &raquo;</li>
        
          <li><a href="../../../index.html">Module code</a> &raquo;</li>
        
      <li>mindspore.nn.wrap.cell_wrapper</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <h1>Source code for mindspore.nn.wrap.cell_wrapper</h1><div class="highlight"><pre>
<span></span><span class="c1"># Copyright 2020 Huawei Technologies Co., Ltd</span>
<span class="c1">#</span>
<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
<span class="c1"># you may not use this file except in compliance with the License.</span>
<span class="c1"># You may obtain a copy of the License at</span>
<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
<span class="c1"># See the License for the specific language governing permissions and</span>
<span class="c1"># limitations under the License.</span>
<span class="c1"># ============================================================================</span>
<span class="sd">&quot;&quot;&quot;Cell_wrapper.&quot;&quot;&quot;</span>
<span class="kn">from</span> <span class="nn">types</span> <span class="kn">import</span> <span class="n">FunctionType</span><span class="p">,</span> <span class="n">MethodType</span>

<span class="kn">from</span> <span class="nn">mindspore.parallel._utils</span> <span class="kn">import</span> <span class="p">(</span><span class="n">_get_device_num</span><span class="p">,</span> <span class="n">_get_gradients_mean</span><span class="p">,</span>
                                       <span class="n">_get_parallel_mode</span><span class="p">,</span> <span class="n">_get_enable_parallel_optimizer</span><span class="p">)</span>
<span class="kn">from</span> <span class="nn">mindspore.context</span> <span class="kn">import</span> <span class="n">ParallelMode</span>
<span class="kn">from</span> <span class="nn">mindspore._checkparam</span> <span class="kn">import</span> <span class="n">Validator</span> <span class="k">as</span> <span class="n">validator</span>
<span class="kn">from</span> <span class="nn">mindspore</span> <span class="kn">import</span> <span class="n">ops</span><span class="p">,</span> <span class="n">nn</span>
<span class="kn">from</span> <span class="nn">...common</span> <span class="kn">import</span> <span class="n">dtype</span> <span class="k">as</span> <span class="n">mstype</span>
<span class="kn">from</span> <span class="nn">...common.parameter</span> <span class="kn">import</span> <span class="n">Parameter</span><span class="p">,</span> <span class="n">ParameterTuple</span>
<span class="kn">from</span> <span class="nn">...ops.primitive</span> <span class="kn">import</span> <span class="n">constexpr</span>
<span class="kn">from</span> <span class="nn">...ops</span> <span class="kn">import</span> <span class="n">composite</span> <span class="k">as</span> <span class="n">C</span>
<span class="kn">from</span> <span class="nn">...ops</span> <span class="kn">import</span> <span class="n">functional</span> <span class="k">as</span> <span class="n">F</span>
<span class="kn">from</span> <span class="nn">...ops</span> <span class="kn">import</span> <span class="n">operations</span> <span class="k">as</span> <span class="n">P</span>
<span class="kn">from</span> <span class="nn">...ops.operations.comm_ops</span> <span class="kn">import</span> <span class="n">_VirtualDataset</span>
<span class="kn">from</span> <span class="nn">..cell</span> <span class="kn">import</span> <span class="n">Cell</span>
<span class="kn">from</span> <span class="nn">.grad_reducer</span> <span class="kn">import</span> <span class="n">DistributedGradReducer</span>

<span class="n">_get_datatype</span> <span class="o">=</span> <span class="n">C</span><span class="o">.</span><span class="n">MultitypeFuncGraph</span><span class="p">(</span><span class="s2">&quot;_get_datatype&quot;</span><span class="p">)</span>


<span class="nd">@_get_datatype</span><span class="o">.</span><span class="n">register</span><span class="p">(</span><span class="s2">&quot;Tensor&quot;</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">_tensors_get_datatype</span><span class="p">(</span><span class="n">param</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Acquire parameter datatype.</span>

<span class="sd">    Args:</span>
<span class="sd">        param (Tensor): The parameter before operation.</span>

<span class="sd">    Returns:</span>
<span class="sd">        mstype, the datatype of parameter.</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">return</span> <span class="n">F</span><span class="o">.</span><span class="n">dtype</span><span class="p">(</span><span class="n">param</span><span class="p">)</span>


<span class="n">_cast_datatype</span> <span class="o">=</span> <span class="n">C</span><span class="o">.</span><span class="n">MultitypeFuncGraph</span><span class="p">(</span><span class="s2">&quot;_cast_datatype&quot;</span><span class="p">)</span>


<span class="nd">@_cast_datatype</span><span class="o">.</span><span class="n">register</span><span class="p">(</span><span class="s2">&quot;TypeType&quot;</span><span class="p">,</span> <span class="s2">&quot;Tensor&quot;</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">_tensors_cast_datatype</span><span class="p">(</span><span class="n">datatype</span><span class="p">,</span> <span class="n">param</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Cast gradient to datatype.</span>

<span class="sd">    Args:</span>
<span class="sd">        datatype (mstype): the destination datatype of parameter.</span>
<span class="sd">        param (Tensor): The parameter before operation.</span>

<span class="sd">    Returns:</span>
<span class="sd">        Tensor, the parameter after operation.</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">return</span> <span class="n">F</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="n">param</span><span class="p">,</span> <span class="n">datatype</span><span class="p">)</span>


<span class="k">class</span> <span class="nc">WithLossCell</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Cell with loss function.</span>

<span class="sd">    Wraps the network with loss function. This Cell accepts data and label as inputs and</span>
<span class="sd">    the computed loss will be returned.</span>

<span class="sd">    Args:</span>
<span class="sd">        backbone (Cell): The target network to wrap.</span>
<span class="sd">        loss_fn (Cell): The loss function used to compute loss.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **data** (Tensor) - Tensor of shape :math:`(N, \ldots)`.</span>
<span class="sd">        - **label** (Tensor) - Tensor of shape :math:`(N, \ldots)`.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, a tensor means the loss value, the shape of which is usually :math:`()`.</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If dtype of `data` or `label` is neither float16 nor float32.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU`` ``CPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; net = Net()</span>
<span class="sd">        &gt;&gt;&gt; loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=False)</span>
<span class="sd">        &gt;&gt;&gt; net_with_criterion = nn.WithLossCell(net, loss_fn)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; batch_size = 2</span>
<span class="sd">        &gt;&gt;&gt; data = Tensor(np.ones([batch_size, 1, 32, 32]).astype(np.float32) * 0.01)</span>
<span class="sd">        &gt;&gt;&gt; label = Tensor(np.ones([batch_size, 10]).astype(np.float32))</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; output_data = net_with_criterion(data, label)</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">backbone</span><span class="p">,</span> <span class="n">loss_fn</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">WithLossCell</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">auto_prefix</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_backbone</span> <span class="o">=</span> <span class="n">backbone</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_loss_fn</span> <span class="o">=</span> <span class="n">loss_fn</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">label</span><span class="p">):</span>
        <span class="n">out</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_backbone</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_loss_fn</span><span class="p">(</span><span class="n">out</span><span class="p">,</span> <span class="n">label</span><span class="p">)</span>

    <span class="nd">@property</span>
    <span class="k">def</span> <span class="nf">backbone_network</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Get the backbone network.</span>

<span class="sd">        Returns:</span>
<span class="sd">            Cell, the backbone network.</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_backbone</span>


<div class="viewcode-block" id="WithGradCell"><a class="viewcode-back" href="../../../../api_python/nn/mindspore.nn.WithGradCell.html#mindspore.nn.WithGradCell">[docs]</a><span class="k">class</span> <span class="nc">WithGradCell</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Cell that returns the gradients.</span>

<span class="sd">    Wraps the network with backward cell to compute gradients. A network with a loss function is necessary</span>
<span class="sd">    as argument. If loss function in None, the network must be a wrapper of network and loss function. This</span>
<span class="sd">    Cell accepts &#39;\*inputs&#39; as inputs and returns gradients for each trainable parameter.</span>

<span class="sd">    Note:</span>
<span class="sd">        Run in PyNative mode.</span>

<span class="sd">    Args:</span>
<span class="sd">        network (Cell): The target network to wrap. The network only supports single output.</span>
<span class="sd">        loss_fn (Cell): Primitive loss function used to compute gradients. Default: None.</span>
<span class="sd">        sens (Union[None, Tensor, Scalar, Tuple ...]): The sensitive for backpropagation, the type and shape</span>
<span class="sd">            must be same as the `network` output. If None, we will fill one to a same type shape of</span>
<span class="sd">            output value. Default: None.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **(\*inputs)** (Tuple(Tensor)) - Tuple of input tensors with shape :math:`(N, \ldots)`.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        list, a list of Tensors with identical shapes as trainable weights.</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If `sens` is not one of None, Tensor, Scalar or Tuple.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU`` ``CPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; # For a defined network Net without loss function</span>
<span class="sd">        &gt;&gt;&gt; net = Net()</span>
<span class="sd">        &gt;&gt;&gt; loss_fn = nn.SoftmaxCrossEntropyWithLogits()</span>
<span class="sd">        &gt;&gt;&gt; grad_net = nn.WithGradCell(net, loss_fn)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # For a network wrapped with loss function</span>
<span class="sd">        &gt;&gt;&gt; net = Net()</span>
<span class="sd">        &gt;&gt;&gt; net_with_criterion = nn.WithLossCell(net, loss_fn)</span>
<span class="sd">        &gt;&gt;&gt; grad_net = nn.WithGradCell(net_with_criterion)</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">network</span><span class="p">,</span> <span class="n">loss_fn</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">sens</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">WithGradCell</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">auto_prefix</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">network</span> <span class="o">=</span> <span class="n">network</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">loss_fn</span> <span class="o">=</span> <span class="n">loss_fn</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">weights</span> <span class="o">=</span> <span class="n">ParameterTuple</span><span class="p">(</span><span class="n">network</span><span class="o">.</span><span class="n">trainable_params</span><span class="p">())</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">grad</span> <span class="o">=</span> <span class="n">C</span><span class="o">.</span><span class="n">GradOperation</span><span class="p">(</span><span class="n">get_by_list</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">sens_param</span><span class="o">=</span><span class="p">(</span><span class="n">sens</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">))</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">sens</span> <span class="o">=</span> <span class="n">sens</span>
        <span class="k">if</span> <span class="n">loss_fn</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">network_with_loss</span> <span class="o">=</span> <span class="n">network</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">network_with_loss</span> <span class="o">=</span> <span class="n">WithLossCell</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">network</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">loss_fn</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">network_with_loss</span><span class="o">.</span><span class="n">set_train</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">inputs</span><span class="p">):</span>
        <span class="n">weights</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">weights</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">sens</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">grads</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">grad</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">network_with_loss</span><span class="p">,</span> <span class="n">weights</span><span class="p">)(</span><span class="o">*</span><span class="n">inputs</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">grads</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">grad</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">network_with_loss</span><span class="p">,</span> <span class="n">weights</span><span class="p">)(</span><span class="o">*</span><span class="n">inputs</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sens</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">grads</span></div>


<span class="k">class</span> <span class="nc">ForwardValueAndGrad</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Encapsulate training network.</span>

<span class="sd">    Including the network and a gradient function. The resulting Cell is trained with input &#39;\*inputs&#39;.</span>
<span class="sd">    The backward graph will be created in the gradient function to calculating gradient.</span>

<span class="sd">    Args:</span>
<span class="sd">        network (Cell): The training network.</span>
<span class="sd">        weights (ParameterTuple): The parameters of the training network that need to calculate the gradient.</span>
<span class="sd">            Default: None.</span>
<span class="sd">        get_all (bool): If True, get all the gradients with respect to inputs. Default: False.</span>
<span class="sd">        get_by_list (bool): If True, get all the gradients with respect to Parameter variables.</span>
<span class="sd">            If get_all and get_by_list are both False, get the gradient with respect to first input.</span>
<span class="sd">            If get_all and get_by_list are both True, get the gradients with respect to inputs and Parameter variables</span>
<span class="sd">            at the same time in the form of ((gradients with respect to inputs),</span>
<span class="sd">            (gradients with respect to parameters)). Default: False.</span>
<span class="sd">        sens_param (bool): Whether to append sensitivity (gradient with respect to output) as input.</span>
<span class="sd">            If sens_param is False, a &#39;ones_like(outputs)&#39; sensitivity will be attached automatically.</span>
<span class="sd">            Default: False.</span>
<span class="sd">            If the sens_param is True, a sensitivity (gradient with respect to output) needs to be transferred through</span>
<span class="sd">            the input parameter.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **(\*inputs)** (Tuple(Tensor...)) - Tuple of inputs with shape :math:`(N, \ldots)`.</span>
<span class="sd">        - **(sens)** - A sensitivity (gradient with respect to output) as the input of backpropagation.</span>
<span class="sd">          If network has single output, the sens is a tensor.</span>
<span class="sd">          If network has multiple outputs, the sens is the tuple(tensor).</span>

<span class="sd">    Outputs:</span>
<span class="sd">        - **forward value** - The result of network forward running.</span>
<span class="sd">        - **gradients** (tuple(tensor)) - The gradients of network parameters and inputs.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU`` ``CPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; import numpy as np</span>
<span class="sd">        &gt;&gt;&gt; from mindspore import Tensor, nn, common, ops, ParameterTuple, Parameter</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; class Net(nn.Cell):</span>
<span class="sd">        ...    def __init__(self):</span>
<span class="sd">        ...        super(Net, self).__init__()</span>
<span class="sd">        ...        self.weight = Parameter(Tensor(np.ones([2, 2]).astype(np.float32)), name=&quot;weight&quot;)</span>
<span class="sd">        ...        self.matmul = ops.MatMul()</span>
<span class="sd">        ...</span>
<span class="sd">        ...    def construct(self, x):</span>
<span class="sd">        ...        out = self.matmul(x, self.weight)</span>
<span class="sd">        ...        return out</span>
<span class="sd">        ...</span>
<span class="sd">        &gt;&gt;&gt; net = Net()</span>
<span class="sd">        &gt;&gt;&gt; criterion = nn.SoftmaxCrossEntropyWithLogits()</span>
<span class="sd">        &gt;&gt;&gt; net_with_criterion = nn.WithLossCell(net, criterion)</span>
<span class="sd">        &gt;&gt;&gt; weight = ParameterTuple(net.trainable_params())</span>
<span class="sd">        &gt;&gt;&gt; train_network = nn.ForwardValueAndGrad(net_with_criterion, weights=weight, get_all=True, get_by_list=True)</span>
<span class="sd">        &gt;&gt;&gt; inputs = Tensor(np.ones([1, 2]).astype(np.float32))</span>
<span class="sd">        &gt;&gt;&gt; labels = Tensor(np.zeros([1, 2]).astype(np.float32))</span>
<span class="sd">        &gt;&gt;&gt; result = train_network(inputs, labels)</span>
<span class="sd">        &gt;&gt;&gt; print(result)</span>
<span class="sd">         (Tensor(shape=[1], dtype=Float32, value= [ 0.00000000e+00]), ((Tensor(shape=[1, 2], dtype=Float32, value=</span>
<span class="sd">        [[ 1.00000000e+00,  1.00000000e+00]]), Tensor(shape=[1, 2], dtype=Float32, value=</span>
<span class="sd">        [[ 0.00000000e+00,  0.00000000e+00]])), (Tensor(shape=[2, 2], dtype=Float32, value=</span>
<span class="sd">        [[ 5.00000000e-01,  5.00000000e-01],</span>
<span class="sd">         [ 5.00000000e-01,  5.00000000e-01]]),)))</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">network</span><span class="p">,</span> <span class="n">weights</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">get_all</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">get_by_list</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">sens_param</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">ForwardValueAndGrad</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">auto_prefix</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">network</span><span class="p">,</span> <span class="p">(</span><span class="n">Cell</span><span class="p">,</span> <span class="n">FunctionType</span><span class="p">,</span> <span class="n">MethodType</span><span class="p">)):</span>
            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;For &#39;ForwardValueAndGrad&#39;, &quot;</span>
                            <span class="sa">f</span><span class="s2">&quot;the argument &#39;network&#39; should be cell, function type or method type, &quot;</span>
                            <span class="sa">f</span><span class="s2">&quot;but got &#39;</span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">network</span><span class="p">)</span><span class="si">}</span><span class="s2">&#39;&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">get_all</span><span class="p">,</span> <span class="nb">bool</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;For &#39;ForwardValueAndGrad&#39;, &quot;</span>
                            <span class="sa">f</span><span class="s2">&quot;the type of &#39;get_all&#39; should be bool, but got &#39;</span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">get_all</span><span class="p">)</span><span class="si">}</span><span class="s2">&#39;&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">get_by_list</span><span class="p">,</span> <span class="nb">bool</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;For &#39;ForwardValueAndGrad&#39;, &quot;</span>
                            <span class="sa">f</span><span class="s2">&quot;the type of &#39;get_by_list&#39; should be bool, but got &#39;</span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">get_by_list</span><span class="p">)</span><span class="si">}</span><span class="s2">&#39;&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">get_by_list</span> <span class="ow">and</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">weights</span><span class="p">,</span> <span class="n">ParameterTuple</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;For &#39;ForwardValueAndGrad&#39;, &quot;</span>
                            <span class="sa">f</span><span class="s2">&quot;when &#39;get_by_list&#39; is set to True, the argument &#39;weights&#39; should be &quot;</span>
                            <span class="sa">f</span><span class="s2">&quot;ParameterTuple type, but got &#39;</span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">weights</span><span class="p">)</span><span class="si">}</span><span class="s2">&#39;&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">network</span> <span class="o">=</span> <span class="n">network</span>
        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">network</span><span class="p">,</span> <span class="n">Cell</span><span class="p">):</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">network</span><span class="o">.</span><span class="n">set_grad</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">weights</span> <span class="o">=</span> <span class="n">weights</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">get_all</span> <span class="o">=</span> <span class="n">get_all</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">get_by_list</span> <span class="o">=</span> <span class="n">get_by_list</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">sens_param</span> <span class="o">=</span> <span class="n">sens_param</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">grad</span> <span class="o">=</span> <span class="n">C</span><span class="o">.</span><span class="n">GradOperation</span><span class="p">(</span><span class="n">get_all</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">get_all</span><span class="p">,</span> <span class="n">get_by_list</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">get_by_list</span><span class="p">,</span> <span class="n">sens_param</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">sens_param</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">inputs</span><span class="p">):</span>
        <span class="n">grad_inputs</span> <span class="o">=</span> <span class="n">inputs</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">sens_param</span><span class="p">:</span>
            <span class="n">inputs</span> <span class="o">=</span> <span class="n">inputs</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
        <span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">network</span><span class="p">(</span><span class="o">*</span><span class="n">inputs</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_by_list</span><span class="p">:</span>
            <span class="n">grads</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">grad</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">network</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">weights</span><span class="p">)(</span><span class="o">*</span><span class="n">grad_inputs</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">grads</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">grad</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">network</span><span class="p">)(</span><span class="o">*</span><span class="n">grad_inputs</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">loss</span><span class="p">,</span> <span class="n">grads</span>


<span class="k">class</span> <span class="nc">TrainOneStepCell</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Network training package class.</span>

<span class="sd">    Wraps the `network` with the `optimizer`. The resulting Cell is trained with input &#39;\*inputs&#39;.</span>
<span class="sd">    The backward graph will be created in the construct function to update the parameter. Different</span>
<span class="sd">    parallel modes are available for training.</span>

<span class="sd">    Args:</span>
<span class="sd">        network (Cell): The training network. The network only supports single output.</span>
<span class="sd">        optimizer (Union[Cell]): Optimizer for updating the network parameters.</span>
<span class="sd">        sens (numbers.Number): The scaling number to be filled as the input of backpropagation. Default value is 1.0.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **(\*inputs)** (Tuple(Tensor)) - Tuple of input tensors with shape :math:`(N, \ldots)`.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, a tensor means the loss value, the shape of which is usually :math:`()`.</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If `sens` is not a numbers.Number.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU`` ``CPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; net = Net()</span>
<span class="sd">        &gt;&gt;&gt; loss_fn = nn.SoftmaxCrossEntropyWithLogits()</span>
<span class="sd">        &gt;&gt;&gt; optim = nn.Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9)</span>
<span class="sd">        &gt;&gt;&gt; #1) Using the WithLossCell provided by MindSpore</span>
<span class="sd">        &gt;&gt;&gt; loss_net = nn.WithLossCell(net, loss_fn)</span>
<span class="sd">        &gt;&gt;&gt; train_net = nn.TrainOneStepCell(loss_net, optim)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; #2) Using user-defined WithLossCell</span>
<span class="sd">        &gt;&gt;&gt; class MyWithLossCell(Cell):</span>
<span class="sd">        ...    def __init__(self, backbone, loss_fn):</span>
<span class="sd">        ...        super(MyWithLossCell, self).__init__(auto_prefix=False)</span>
<span class="sd">        ...        self._backbone = backbone</span>
<span class="sd">        ...        self._loss_fn = loss_fn</span>
<span class="sd">        ...</span>
<span class="sd">        ...    def construct(self, x, y, label):</span>
<span class="sd">        ...        out = self._backbone(x, y)</span>
<span class="sd">        ...        return self._loss_fn(out, label)</span>
<span class="sd">        ...</span>
<span class="sd">        ...    @property</span>
<span class="sd">        ...    def backbone_network(self):</span>
<span class="sd">        ...        return self._backbone</span>
<span class="sd">        ...</span>
<span class="sd">        &gt;&gt;&gt; loss_net = MyWithLossCell(net, loss_fn)</span>
<span class="sd">        &gt;&gt;&gt; train_net = nn.TrainOneStepCell(loss_net, optim)</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">network</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">,</span> <span class="n">sens</span><span class="o">=</span><span class="mf">1.0</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">TrainOneStepCell</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">auto_prefix</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">network</span> <span class="o">=</span> <span class="n">network</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">network</span><span class="o">.</span><span class="n">set_grad</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span> <span class="o">=</span> <span class="n">optimizer</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">weights</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">parameters</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">grad</span> <span class="o">=</span> <span class="n">C</span><span class="o">.</span><span class="n">GradOperation</span><span class="p">(</span><span class="n">get_by_list</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">sens_param</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">sens</span> <span class="o">=</span> <span class="n">sens</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">reducer_flag</span> <span class="o">=</span> <span class="kc">False</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">grad_reducer</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">identity</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">parallel_mode</span> <span class="o">=</span> <span class="n">_get_parallel_mode</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">reducer_flag</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parallel_mode</span> <span class="ow">in</span> <span class="p">(</span><span class="n">ParallelMode</span><span class="o">.</span><span class="n">DATA_PARALLEL</span><span class="p">,</span> <span class="n">ParallelMode</span><span class="o">.</span><span class="n">HYBRID_PARALLEL</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">reducer_flag</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">mean</span> <span class="o">=</span> <span class="n">_get_gradients_mean</span><span class="p">()</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">degree</span> <span class="o">=</span> <span class="n">_get_device_num</span><span class="p">()</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">grad_reducer</span> <span class="o">=</span> <span class="n">DistributedGradReducer</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">weights</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">mean</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">degree</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">inputs</span><span class="p">):</span>
        <span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">network</span><span class="p">(</span><span class="o">*</span><span class="n">inputs</span><span class="p">)</span>
        <span class="n">sens</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">fill</span><span class="p">(</span><span class="n">loss</span><span class="o">.</span><span class="n">dtype</span><span class="p">,</span> <span class="n">loss</span><span class="o">.</span><span class="n">shape</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sens</span><span class="p">)</span>
        <span class="n">grads</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">grad</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">network</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">weights</span><span class="p">)(</span><span class="o">*</span><span class="n">inputs</span><span class="p">,</span> <span class="n">sens</span><span class="p">)</span>
        <span class="n">grads</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">grad_reducer</span><span class="p">(</span><span class="n">grads</span><span class="p">)</span>
        <span class="n">loss</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">depend</span><span class="p">(</span><span class="n">loss</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="p">(</span><span class="n">grads</span><span class="p">))</span>
        <span class="k">return</span> <span class="n">loss</span>


<span class="k">class</span> <span class="nc">GetNextSingleOp</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Cell to run for getting the next operation.</span>

<span class="sd">    For detailed information, refer to `ops.operations.GetNext`.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_types (list[:class:`mindspore.dtype`]): The types of dataset.</span>
<span class="sd">        dataset_shapes (list[tuple[int]]): The shapes of dataset.</span>
<span class="sd">        queue_name (str): Queue name to fetch the data.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        No inputs.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        tuple[Tensor], the data get from Dataset.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; import mindspore</span>
<span class="sd">        &gt;&gt;&gt; from mindspore import ops, nn</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; train_dataset = create_custom_dataset()</span>
<span class="sd">        &gt;&gt;&gt; dataset_helper = mindspore.DatasetHelper(train_dataset, dataset_sink_mode=True)</span>
<span class="sd">        &gt;&gt;&gt; dataset = dataset_helper.iter.dataset</span>
<span class="sd">        &gt;&gt;&gt; dataset_types, dataset_shapes = dataset_helper.types_shapes()</span>
<span class="sd">        &gt;&gt;&gt; queue_name = dataset.__transfer_dataset__.queue_name</span>
<span class="sd">        &gt;&gt;&gt; get_next_single_op_net = nn.GetNextSingleOp(dataset_types, dataset_shapes, queue_name)</span>
<span class="sd">        &gt;&gt;&gt; data, label = get_next_single_op_net()</span>
<span class="sd">        &gt;&gt;&gt; relu = ops.ReLU()</span>
<span class="sd">        &gt;&gt;&gt; result = relu(data).asnumpy()</span>
<span class="sd">        &gt;&gt;&gt; print(result.shape)</span>
<span class="sd">        (32, 1, 32, 32)</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_types</span><span class="p">,</span> <span class="n">dataset_shapes</span><span class="p">,</span> <span class="n">queue_name</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">GetNextSingleOp</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">get_next</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">GetNext</span><span class="p">(</span><span class="n">dataset_types</span><span class="p">,</span> <span class="n">dataset_shapes</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">dataset_types</span><span class="p">),</span> <span class="n">queue_name</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_next</span><span class="p">()</span>


<span class="k">class</span> <span class="nc">_VirtualDatasetCell</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Wrap the network with virtual dataset to convert data parallel layout to model parallel layout.</span>

<span class="sd">    _VirtualDataset is a virtual Primitive, it does not exist in the final executing graph. Inputs and outputs</span>
<span class="sd">    of _VirtualDataset are distributed in data parallel pattern, tensor redistribution Primitives is inserted</span>
<span class="sd">    dynamically during the graph compile process.</span>

<span class="sd">    Note:</span>
<span class="sd">        Only used in semi auto parallel and auto parallel mode.</span>

<span class="sd">    Args:</span>
<span class="sd">        backbone (Cell): The target network to wrap.</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; net = Net()</span>
<span class="sd">        &gt;&gt;&gt; net = _VirtualDatasetCell(net)</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">backbone</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">_VirtualDatasetCell</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">auto_prefix</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_backbone</span> <span class="o">=</span> <span class="n">backbone</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_virtual_dataset</span> <span class="o">=</span> <span class="n">_VirtualDataset</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">inputs</span><span class="p">):</span>
        <span class="n">output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_virtual_dataset</span><span class="p">(</span><span class="o">*</span><span class="n">inputs</span><span class="p">)</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_backbone</span><span class="p">(</span><span class="o">*</span><span class="n">output</span><span class="p">)</span>


<span class="nd">@constexpr</span>
<span class="k">def</span> <span class="nf">_check_shape_value_on_axis_divided_by_target_value</span><span class="p">(</span><span class="n">input_shape</span><span class="p">,</span> <span class="n">dim</span><span class="p">,</span> <span class="n">param_name</span><span class="p">,</span> <span class="n">cls_name</span><span class="p">,</span> <span class="n">target_value</span><span class="p">):</span>
    <span class="k">if</span> <span class="n">input_shape</span><span class="p">[</span><span class="n">dim</span><span class="p">]</span> <span class="o">%</span> <span class="n">target_value</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">cls_name</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">param_name</span><span class="si">}</span><span class="s2"> at </span><span class="si">{</span><span class="n">dim</span><span class="si">}</span><span class="s2"> shape should be divided by </span><span class="si">{</span><span class="n">target_value</span><span class="si">}</span><span class="s2">,&quot;</span>
                         <span class="sa">f</span><span class="s2">&quot;but got </span><span class="si">{</span><span class="n">input_shape</span><span class="p">[</span><span class="n">dim</span><span class="p">]</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
    <span class="k">return</span> <span class="kc">True</span>


<span class="k">class</span> <span class="nc">_MicroBatch</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    transform mini-batch to micro-batch in pipeline parallel.</span>

<span class="sd">    Args:</span>
<span class="sd">       params (micro_size): The number of micro-batch.</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">micro_size</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">_MicroBatch</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">shape</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Shape</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">micro_size</span> <span class="o">=</span> <span class="n">micro_size</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">strided_slice</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">StridedSlice</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="o">*</span><span class="n">inputs</span><span class="p">):</span>
        <span class="n">micro_inputs</span> <span class="o">=</span> <span class="p">()</span>
        <span class="k">for</span> <span class="n">each_input</span> <span class="ow">in</span> <span class="n">inputs</span><span class="p">:</span>
            <span class="n">input_shape</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">shape</span><span class="p">(</span><span class="n">each_input</span><span class="p">)</span>
            <span class="n">_check_shape_value_on_axis_divided_by_target_value</span><span class="p">(</span><span class="n">input_shape</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="s2">&quot;inputs&quot;</span><span class="p">,</span>
                                                               <span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">micro_size</span><span class="p">)</span>
            <span class="n">micro_batch_begin</span> <span class="o">=</span> <span class="n">i</span> <span class="o">*</span> <span class="n">input_shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">//</span> <span class="bp">self</span><span class="o">.</span><span class="n">micro_size</span>
            <span class="n">micro_batch_end</span> <span class="o">=</span> <span class="p">(</span><span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">input_shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">//</span> <span class="bp">self</span><span class="o">.</span><span class="n">micro_size</span>
            <span class="n">strided_slice_begin</span> <span class="o">=</span> <span class="p">(</span><span class="n">micro_batch_begin</span><span class="p">,)</span>
            <span class="n">strided_slice_strides</span> <span class="o">=</span> <span class="p">(</span><span class="mi">1</span><span class="p">,)</span>
            <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">input_shape</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span><span class="p">):</span>
                <span class="n">strided_slice_begin</span> <span class="o">+=</span> <span class="p">(</span><span class="mi">0</span><span class="p">,)</span>
                <span class="n">strided_slice_strides</span> <span class="o">+=</span> <span class="p">(</span><span class="mi">1</span><span class="p">,)</span>
            <span class="n">strided_slice_end</span> <span class="o">=</span> <span class="p">(</span><span class="n">micro_batch_end</span><span class="p">,)</span>
            <span class="n">strided_slice_end</span> <span class="o">+=</span> <span class="n">input_shape</span><span class="p">[</span><span class="mi">1</span><span class="p">:]</span>
            <span class="n">micro_input</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">strided_slice</span><span class="p">(</span><span class="n">each_input</span><span class="p">,</span> <span class="n">strided_slice_begin</span><span class="p">,</span> <span class="n">strided_slice_end</span><span class="p">,</span> <span class="n">strided_slice_strides</span><span class="p">)</span>
            <span class="n">micro_inputs</span> <span class="o">+=</span> <span class="p">(</span><span class="n">micro_input</span><span class="p">,)</span>
        <span class="k">return</span> <span class="n">micro_inputs</span>


<div class="viewcode-block" id="MicroBatchInterleaved"><a class="viewcode-back" href="../../../../api_python/nn/mindspore.nn.MicroBatchInterleaved.html#mindspore.nn.MicroBatchInterleaved">[docs]</a><span class="k">class</span> <span class="nc">MicroBatchInterleaved</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Wrap the network with Batch Size.</span>

<span class="sd">    Args:</span>
<span class="sd">        network (Cell): The target network to wrap.</span>
<span class="sd">        interleave_num (int): split num of batch size. Default: 2.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; net = Net()</span>
<span class="sd">        &gt;&gt;&gt; net = MicroBatchInterleaved(net, 4)</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">network</span><span class="p">,</span> <span class="n">interleave_num</span><span class="o">=</span><span class="mi">2</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">MicroBatchInterleaved</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">auto_prefix</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">interleave_num</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">&quot;For &#39;MicroBatchInterleaved&#39;, the argument &#39;interleave_num&#39; should be integer, &quot;</span>
                            <span class="s2">&quot;but got the type : </span><span class="si">{}</span><span class="s2">.&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">interleave_num</span><span class="p">)))</span>
        <span class="k">if</span> <span class="n">interleave_num</span> <span class="o">&lt;=</span> <span class="mi">0</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;For &#39;MicroBatchInterleaved&#39;, the argument &#39;interleave_num&#39; should be greater than 0, &quot;</span>
                             <span class="s2">&quot;but got </span><span class="si">{}</span><span class="s2">.&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">interleave_num</span><span class="p">))</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">network</span> <span class="o">=</span> <span class="n">network</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">interleave_num</span> <span class="o">=</span> <span class="n">interleave_num</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">interleave_inputs</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">CellList</span><span class="p">()</span>
        <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">interleave_num</span><span class="p">):</span>
            <span class="n">interleave_data</span> <span class="o">=</span> <span class="n">_MicroBatch</span><span class="p">(</span><span class="n">interleave_num</span><span class="p">)</span>
            <span class="n">interleave_data</span><span class="o">.</span><span class="n">strided_slice</span><span class="o">.</span><span class="n">add_prim_attr</span><span class="p">(</span><span class="s2">&quot;strided_slice_flag&quot;</span><span class="p">,</span> <span class="kc">True</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">interleave_inputs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">interleave_data</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">inputs</span><span class="p">):</span>
        <span class="n">output</span> <span class="o">=</span> <span class="mf">0.0</span>
        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">interleave_num</span><span class="p">):</span>
            <span class="n">interleave_input</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">interleave_inputs</span><span class="p">[</span><span class="n">i</span><span class="p">](</span><span class="n">i</span><span class="p">,</span> <span class="o">*</span><span class="n">inputs</span><span class="p">)</span>
            <span class="n">output</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">network</span><span class="p">(</span><span class="o">*</span><span class="n">interleave_input</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">output</span></div>


<span class="k">class</span> <span class="nc">PipelineCell</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Wrap the network with Micro Batch.</span>

<span class="sd">    Note:</span>
<span class="sd">        micro_size must be greater or equal to pipeline stages.</span>

<span class="sd">    Args:</span>
<span class="sd">        network (Cell): The target network to wrap.</span>
<span class="sd">        micro_size (int): MicroBatch size.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; net = Net()</span>
<span class="sd">        &gt;&gt;&gt; net = PipelineCell(net, 4)</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">network</span><span class="p">,</span> <span class="n">micro_size</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">PipelineCell</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">auto_prefix</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">network</span> <span class="o">=</span> <span class="n">network</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">micro_inputs</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">CellList</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">micro_size</span> <span class="o">=</span> <span class="n">micro_size</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">add_list</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">micro_size</span><span class="p">):</span>
            <span class="n">micro_input</span> <span class="o">=</span> <span class="n">_MicroBatch</span><span class="p">(</span><span class="n">micro_size</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">micro_inputs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">micro_input</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">add</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Add</span><span class="p">()</span><span class="o">.</span><span class="n">add_prim_attr</span><span class="p">(</span><span class="s2">&quot;pipeline_end&quot;</span><span class="p">,</span> <span class="n">i</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">add_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">add</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">inputs</span><span class="p">):</span>
        <span class="n">ret</span> <span class="o">=</span> <span class="kc">None</span>
        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">micro_size</span><span class="p">):</span>
            <span class="n">micro_input</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">micro_inputs</span><span class="p">[</span><span class="n">i</span><span class="p">](</span><span class="n">i</span><span class="p">,</span> <span class="o">*</span><span class="n">inputs</span><span class="p">)</span>
            <span class="n">output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">network</span><span class="p">(</span><span class="o">*</span><span class="n">micro_input</span><span class="p">)</span>
            <span class="k">if</span> <span class="n">ret</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="n">ret</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">add_list</span><span class="p">[</span><span class="n">i</span><span class="p">](</span><span class="n">ret</span><span class="p">,</span> <span class="n">output</span><span class="p">)</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">ret</span> <span class="o">=</span> <span class="n">output</span>
        <span class="k">return</span> <span class="n">ret</span>


<span class="k">def</span> <span class="nf">_pipeline_clear_grad</span><span class="p">(</span><span class="n">accu_grad</span><span class="p">,</span> <span class="n">grad</span><span class="p">):</span>
    <span class="n">accu_grad</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">depend</span><span class="p">(</span><span class="n">accu_grad</span><span class="p">,</span> <span class="n">grad</span><span class="p">)</span>
    <span class="n">zeros</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">tensor_mul</span><span class="p">(</span><span class="n">accu_grad</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">F</span><span class="o">.</span><span class="n">assign</span><span class="p">(</span><span class="n">accu_grad</span><span class="p">,</span> <span class="n">zeros</span><span class="p">)</span>


<span class="k">class</span> <span class="nc">_TrainPipelineAccuStepCell</span><span class="p">(</span><span class="n">TrainOneStepCell</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Wraps the network with an optimizer in pipeline mode.</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">network</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">,</span> <span class="n">sens</span><span class="o">=</span><span class="mf">1.0</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">_TrainPipelineAccuStepCell</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">network</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">,</span> <span class="n">sens</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">accu_grads</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">weights</span><span class="o">.</span><span class="n">clone</span><span class="p">(</span><span class="n">prefix</span><span class="o">=</span><span class="s2">&quot;accu_grads&quot;</span><span class="p">,</span> <span class="n">init</span><span class="o">=</span><span class="s2">&quot;zeros&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">hyper_map</span> <span class="o">=</span> <span class="n">ops</span><span class="o">.</span><span class="n">HyperMap</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">opt_shard</span> <span class="o">=</span> <span class="n">_get_enable_parallel_optimizer</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">inputs</span><span class="p">):</span>
        <span class="n">weights</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">weights</span>
        <span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">network</span><span class="p">(</span><span class="o">*</span><span class="n">inputs</span><span class="p">)</span>
        <span class="n">sens</span> <span class="o">=</span> <span class="n">ops</span><span class="o">.</span><span class="n">Fill</span><span class="p">()(</span><span class="n">ops</span><span class="o">.</span><span class="n">DType</span><span class="p">()(</span><span class="n">loss</span><span class="p">),</span> <span class="n">ops</span><span class="o">.</span><span class="n">Shape</span><span class="p">()(</span><span class="n">loss</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">sens</span><span class="p">)</span>
        <span class="n">grads</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">grad</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">network</span><span class="p">,</span> <span class="n">weights</span><span class="p">)(</span><span class="o">*</span><span class="n">inputs</span><span class="p">,</span> <span class="n">sens</span><span class="p">)</span>
        <span class="n">accu_grads</span> <span class="o">=</span> <span class="n">ops</span><span class="o">.</span><span class="n">depend</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">accu_grads</span><span class="p">,</span> <span class="n">grads</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">opt_shard</span><span class="p">:</span>
            <span class="n">succ</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="p">(</span><span class="n">grads</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">succ</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="p">(</span><span class="n">accu_grads</span><span class="p">)</span>
        <span class="n">loss</span> <span class="o">=</span> <span class="n">ops</span><span class="o">.</span><span class="n">depend</span><span class="p">(</span><span class="n">loss</span><span class="p">,</span> <span class="n">succ</span><span class="p">)</span>
        <span class="n">clear</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">hyper_map</span><span class="p">(</span><span class="n">_pipeline_clear_grad</span><span class="p">,</span> <span class="n">accu_grads</span><span class="p">,</span> <span class="n">grads</span><span class="p">)</span>
        <span class="n">loss</span> <span class="o">=</span> <span class="n">ops</span><span class="o">.</span><span class="n">depend</span><span class="p">(</span><span class="n">loss</span><span class="p">,</span> <span class="n">clear</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">loss</span>


<span class="k">class</span> <span class="nc">VirtualDatasetCellTriple</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Wrap the network with virtual dataset to convert data parallel layout to model parallel layout.</span>

<span class="sd">    VirtualDatasetCellTriple is a virtual Primitive, it does not exist in the final executing graph. Inputs and outputs</span>
<span class="sd">    of VirtualDatasetCellTriple are distributed in data parallel pattern, tensor redistribution Primitives is inserted</span>
<span class="sd">    dynamically during the graph compile process.</span>

<span class="sd">    Note:</span>
<span class="sd">        Only used in semi auto parallel and auto parallel mode. There are three inputs, as contrary to two inputs in</span>
<span class="sd">        _VirtualDatasetCell.</span>

<span class="sd">    Args:</span>
<span class="sd">        backbone (Cell): The target network to wrap.</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; net = Net()</span>
<span class="sd">        &gt;&gt;&gt; net = VirtualDatasetCellTriple(net)</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">backbone</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">VirtualDatasetCellTriple</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">auto_prefix</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_backbone</span> <span class="o">=</span> <span class="n">backbone</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_virtual_dataset</span> <span class="o">=</span> <span class="n">_VirtualDataset</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">a</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="n">c</span><span class="p">):</span>
        <span class="n">a_</span><span class="p">,</span> <span class="n">b_</span><span class="p">,</span> <span class="n">c_</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_virtual_dataset</span><span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="n">c</span><span class="p">)</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_backbone</span><span class="p">(</span><span class="n">a_</span><span class="p">,</span> <span class="n">b_</span><span class="p">,</span> <span class="n">c_</span><span class="p">)</span>


<span class="k">class</span> <span class="nc">WithEvalCell</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Wraps the forward network with the loss function.</span>

<span class="sd">    It returns loss, forward output and label to calculate the metrics.</span>

<span class="sd">    Args:</span>
<span class="sd">        network (Cell): The forward network.</span>
<span class="sd">        loss_fn (Cell): The loss function.</span>
<span class="sd">        add_cast_fp32 (bool): Whether to adjust the data type to float32. Default: False.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **data** (Tensor) - Tensor of shape :math:`(N, \ldots)`.</span>
<span class="sd">        - **label** (Tensor) - Tensor of shape :math:`(N, \ldots)`.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tuple(Tensor), containing a scalar loss Tensor, a network output Tensor of shape :math:`(N, \ldots)`</span>
<span class="sd">        and a label Tensor of shape :math:`(N, \ldots)`.</span>

<span class="sd">    Raises:</span>
<span class="sd">        TypeError: If `add_cast_fp32` is not a bool.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU`` ``CPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; # Forward network without loss function</span>
<span class="sd">        &gt;&gt;&gt; net = Net()</span>
<span class="sd">        &gt;&gt;&gt; loss_fn = nn.SoftmaxCrossEntropyWithLogits()</span>
<span class="sd">        &gt;&gt;&gt; eval_net = nn.WithEvalCell(net, loss_fn)</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">network</span><span class="p">,</span> <span class="n">loss_fn</span><span class="p">,</span> <span class="n">add_cast_fp32</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">WithEvalCell</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">auto_prefix</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_network</span> <span class="o">=</span> <span class="n">network</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_loss_fn</span> <span class="o">=</span> <span class="n">loss_fn</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">add_cast_fp32</span> <span class="o">=</span> <span class="n">validator</span><span class="o">.</span><span class="n">check_value_type</span><span class="p">(</span><span class="s2">&quot;add_cast_fp32&quot;</span><span class="p">,</span> <span class="n">add_cast_fp32</span><span class="p">,</span> <span class="p">[</span><span class="nb">bool</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">cls_name</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">label</span><span class="p">):</span>
        <span class="n">outputs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_network</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">add_cast_fp32</span><span class="p">:</span>
            <span class="n">label</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">mixed_precision_cast</span><span class="p">(</span><span class="n">mstype</span><span class="o">.</span><span class="n">float32</span><span class="p">,</span> <span class="n">label</span><span class="p">)</span>
            <span class="n">outputs</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="n">outputs</span><span class="p">,</span> <span class="n">mstype</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>
        <span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_loss_fn</span><span class="p">(</span><span class="n">outputs</span><span class="p">,</span> <span class="n">label</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">loss</span><span class="p">,</span> <span class="n">outputs</span><span class="p">,</span> <span class="n">label</span>


<span class="k">class</span> <span class="nc">ParameterUpdate</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Cell that updates parameter.</span>

<span class="sd">    With this Cell, one can manually update `param` with the input `Tensor`.</span>

<span class="sd">    Args:</span>
<span class="sd">        param (Parameter): The parameter to be updated manually.</span>

<span class="sd">    Inputs:</span>
<span class="sd">        - **x** (Tensor) - A tensor whose shape and type are the same with `param`.</span>

<span class="sd">    Outputs:</span>
<span class="sd">        Tensor, the input `x`.</span>

<span class="sd">    Raises:</span>
<span class="sd">        KeyError: If parameter with the specified name does not exist.</span>

<span class="sd">    Supported Platforms:</span>
<span class="sd">        ``Ascend`` ``GPU`` ``CPU``</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; import numpy as np</span>
<span class="sd">        &gt;&gt;&gt; import mindspore</span>
<span class="sd">        &gt;&gt;&gt; from mindspore import nn, Tensor</span>
<span class="sd">        &gt;&gt;&gt; network = nn.Dense(3, 4)</span>
<span class="sd">        &gt;&gt;&gt; param = network.parameters_dict()[&#39;weight&#39;]</span>
<span class="sd">        &gt;&gt;&gt; update = nn.ParameterUpdate(param)</span>
<span class="sd">        &gt;&gt;&gt; update.phase = &quot;update_param&quot;</span>
<span class="sd">        &gt;&gt;&gt; weight = Tensor(np.arange(12).reshape((4, 3)), mindspore.float32)</span>
<span class="sd">        &gt;&gt;&gt; output = update(weight)</span>
<span class="sd">        &gt;&gt;&gt; print(output)</span>
<span class="sd">        [[ 0.  1.  2.]</span>
<span class="sd">         [ 3.  4.  5.]</span>
<span class="sd">         [ 6.  7.  8.]</span>
<span class="sd">         [ 9. 10. 11.]]</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">param</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">ParameterUpdate</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">auto_prefix</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">param</span><span class="p">,</span> <span class="n">Parameter</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">&quot;For &#39;ParameterUpdate&#39;, &#39;param&#39; must be &#39;Parameter&#39;, but got </span><span class="si">{}</span><span class="s2">.&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">param</span><span class="p">)))</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_param</span> <span class="o">=</span> <span class="n">param</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="n">F</span><span class="o">.</span><span class="n">assign</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_param</span><span class="p">,</span> <span class="n">x</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">x</span>


<span class="k">class</span> <span class="nc">_BroadCastCell</span><span class="p">(</span><span class="n">Cell</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Broadcast the parameters from device 0 to other devices.</span>

<span class="sd">    Args:</span>
<span class="sd">       params (list): The parameters of Net.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">_BroadCastCell</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="kn">from</span> <span class="nn">mindspore.communication.management</span> <span class="kn">import</span> <span class="n">get_group_size</span><span class="p">,</span> <span class="n">create_group</span>
        <span class="kn">from</span> <span class="nn">mindspore</span> <span class="kn">import</span> <span class="n">context</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">map_</span> <span class="o">=</span> <span class="n">C</span><span class="o">.</span><span class="n">Map</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">params</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">params</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">context</span><span class="o">.</span><span class="n">get_context</span><span class="p">(</span><span class="s2">&quot;device_target&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="s2">&quot;Ascend&quot;</span> <span class="ow">and</span> <span class="n">context</span><span class="o">.</span><span class="n">get_context</span><span class="p">(</span><span class="s2">&quot;mode&quot;</span><span class="p">)</span> <span class="o">!=</span> <span class="n">context</span><span class="o">.</span><span class="n">PYNATIVE_MODE</span><span class="p">:</span>
            <span class="n">rank_list</span> <span class="o">=</span> <span class="p">[</span><span class="nb">id</span> <span class="k">for</span> <span class="nb">id</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">get_group_size</span><span class="p">())]</span>
            <span class="n">create_group</span><span class="p">(</span><span class="s2">&quot;BroadcastWorldGroup&quot;</span><span class="p">,</span> <span class="n">rank_list</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">broadcast</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Broadcast</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">group</span><span class="o">=</span><span class="s2">&quot;BroadcastWorldGroup&quot;</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">broadcast</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">Broadcast</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">construct</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="n">datatypes</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">map_</span><span class="p">(</span><span class="n">F</span><span class="o">.</span><span class="n">partial</span><span class="p">(</span><span class="n">_get_datatype</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">params</span><span class="p">)</span>
        <span class="n">params</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">map_</span><span class="p">(</span><span class="n">F</span><span class="o">.</span><span class="n">partial</span><span class="p">(</span><span class="n">_cast_datatype</span><span class="p">,</span> <span class="n">mstype</span><span class="o">.</span><span class="n">float32</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">params</span><span class="p">)</span>
        <span class="n">params</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">broadcast</span><span class="p">(</span><span class="n">params</span><span class="p">)</span>
        <span class="n">new_params</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">map_</span><span class="p">(</span><span class="n">F</span><span class="o">.</span><span class="n">partial</span><span class="p">(</span><span class="n">_cast_datatype</span><span class="p">),</span> <span class="n">datatypes</span><span class="p">,</span> <span class="n">params</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">new_params</span>
</pre></div>

           </div>
           
          </div>
          <footer>

  <hr/>

  <div role="contentinfo">
    <p>
        &#169; Copyright 2021, MindSpore.

    </p>
  </div>
    
    
    
    Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
    
    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
    
    provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>
        </div>
      </div>

    </section>

  </div>
  

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
  
    
   

</body>
</html>